#!/usr/bin/env python

# (c) 2007-2011 Helsinki University of Technology and University of Helsinki
# Licensed under the MIT license; see LICENSE.txt for more information.

from worker import Worker
from document import Document

from pysqlite2 import dbapi2 as sqlite
import os
import os.path
import rdf
import sys

# import Kid templates
import kid
kid.enable_import()
from harava.template import document
from harava.template import documents_failed
from harava.template import documents_skipped
from harava.template import documents_succesful
from harava.template import documents_warnings
from harava.template import index
from harava.template import problems_phase
from harava.template import problems_severity

# levels
LEVELS = [ 'SKIP', 'CRIT', 'WARN', 'NOTE', 'SUCC' ]

# phases
PHASES = [ 'fetch', 'parse', 'import', 'map', 'validate', 'filter', 'fulltext', 'content', 'preview', 'store' ]


class Report (Worker):
  def __init__(self, session, reportlevel, single=False):
    Worker.__init__(self, session.getFatalEvent())
    self._session = session
    self._reportlevel = reportlevel
    self._single = single
    self._docs = {}	# url -> { 'id': MD5string, 'title': title }

    if not single:
      reportdir = self._session.getConfig().getReportDir()
      if not os.path.exists(reportdir):
        os.mkdir(reportdir)
      
      # check for existence of the static/ directory with images, js and CSS;
      # create and populate it if it doesn't exist yet
      if not os.path.exists(reportdir + "/static"):
        # extract the static files from the egg
        from pkg_resources import Requirement, resource_filename
        import shutil
        staticdir = resource_filename(Requirement.parse('harava'),'report-static')
        shutil.copytree(staticdir, reportdir + '/static')

      sourcename = self._session.getSource().getName()
      self._reportpath = "%s/%s" % (reportdir, sourcename)
      if not os.path.exists(self._reportpath):
        os.mkdir(self._reportpath)
    else:
      self._reportpath = None # avoid AttributeError

  
  def initialize(self):
    self._conn = sqlite.connect(':memory:')
#    if os.path.exists('test.db'):
#      os.unlink('test.db')
#    self._conn = sqlite.connect('test.db')
    
    # create the report tables
    cursor = self._conn.cursor()
    cursor.execute("""
      create table problemtype (
        id integer primary key,
        level integer,
        phase integer,
        type varchar
      )""")
    cursor.execute("create index ptkey on problemtype(level, phase, type)")
    cursor.execute("""
      create table detail (
        id integer primary key,
        problemtype integer,
        details varchar
      )""")
    cursor.execute("create index detkey on detail(problemtype, details)")
    cursor.execute("""
      create table document (
        id integer primary key,
        url varchar
      )""")
    cursor.execute("create index dockey on document(url)")
    cursor.execute("""
      create table documentdetail (
        id integer primary key,
        detail integer,
        document integer
      )""")
    cursor.execute("create index docdetkey on documentdetail(detail,document)")
    
    self._conn.commit()
    
  def log(self, level, phase, type, details=None, doc=None):
    """ log an event, but pass it on to another thread for actual handling """
    item = (level, phase, type, details, doc)
    self.addWork(item)
  
  def createDocumentReport(self, doc):
    # make a note of the Document object so it can be retrieved by url
    self._docs[doc.getURL()] = {
      'id': doc.getID(),
      'title': doc.getTitle() or doc.getURL()
    }
    doc.releaseData() # just to be sure the doc data has really been released by now!

    if self._reportlevel < 2 and not self._single: return
    
    self.addWork(doc)
    
  
  def process(self, item):
    # it's a finished document - write a report
    if isinstance(item, Document):
      self._createDocumentReport(self._reportpath, item)
      return True
    
    # if it wasn't a document, it's a log message
    level, phase, type, details, doc = item
    
    # translate level and phase to indices so they sort properly (in ORDER BY clauses)
    level = LEVELS.index(level)
    phase = PHASES.index(phase)
    
    # fixup possible None values
    if details is None: details = ''
    # make sure it's a real unicode obj
    details=unicode(details)
    
    cursor = self._conn.cursor()
    
    # see if problemtype already exists; add it if necessary
    cursor.execute("select id from problemtype where level=? and phase=? and type=?", (level, phase, type))
    ret = cursor.fetchone()
    if ret is not None:
      ptid = ret[0]
    else:
      cursor.execute("insert into problemtype (id, level, phase, type) values (null,?,?,?)", (level, phase, type))
      ptid = cursor.lastrowid
    
    # see if detail already exists; add it if necessary
    cursor.execute("select id from detail where problemtype=? and details=?", (ptid, details))
    ret = cursor.fetchone()
    if ret is not None:
      detid = ret[0]
    else:
      cursor.execute("insert into detail (id, problemtype, details) values (null,?,?)", (ptid, details))
      detid = cursor.lastrowid
    
    # see if document already exists; add it if necessary
    cursor.execute("select id from document where url=?", (doc.getURL(),))
    ret = cursor.fetchone()
    if ret is not None:
      docid = ret[0]
    else:
      cursor.execute("insert into document (id,url) values (null,?)", (doc.getURL(),))
      docid = cursor.lastrowid
    
    # add a log message to documentdetail
    cursor.execute("insert into documentdetail (id, detail, document) values (null,?,?)", (detid, docid))
    
    self._conn.commit()
    
    return True

  def _createReport(self, reportpath, report, name):
#    template = kid.Template(file=TEMPLATEDIR + '/%s.kid' % name, rep=self)
    template = report.Template(rep=self)
    filename = "%s/%s.html" % (reportpath, name)
    template.write(filename, output='xhtml-strict', format='compact')
  
  def _createDocumentReport(self, reportpath, doc):
#    template = kid.Template(file=TEMPLATEDIR + '/document.kid', rep=self, doc=doc)
    template = document.Template(rep=self, doc=doc)
    if self._single:
      file = sys.stdout
    else:
      file = "%s/doc_%s.html" % (reportpath, doc.getID())
    template.write(file, output='xhtml-strict', format='compact')
  
  def finish(self):
    if self._reportlevel < 1 or self._single: return
  
    self._createReport(self._reportpath, index, 'index')
    self._createReport(self._reportpath, problems_severity, 'problems-severity')
    self._createReport(self._reportpath, problems_phase, 'problems-phase')
    self._createReport(self._reportpath, documents_succesful, 'documents-succesful')
    self._createReport(self._reportpath, documents_warnings, 'documents-warnings')
    self._createReport(self._reportpath, documents_failed, 'documents-failed')
    self._createReport(self._reportpath, documents_skipped, 'documents-skipped')

  def getSource(self):
    return self._session.getSource()

  def _getProblem(self, level, phase, type, detid, details, ndocs):
    cursor = self._conn.cursor()
    cursor.execute("select url from document, documentdetail where documentdetail.document=document.id and detail=?", (detid,))
    docurls = [docrow[0] for docrow in cursor.fetchall()]
    docs = [self._docs[url] for url in docurls[:10]]
    return {'level': LEVELS[level], 'phase': PHASES[phase], 'type': type, 'details': details, 'ndocs': ndocs, 'docs': docs}
  
  def getProblemsBySeverity(self):
    """ yield tuples of the form (severity, [{problem}, {problem}...]) """

    cursor = self._conn.cursor()

    for levid, level in enumerate(LEVELS):
      cursor.execute("""
        select phase, type, detail.id, details, count(document) as ndocs
        from problemtype, detail, documentdetail
        where detail.problemtype = problemtype.id
          and documentdetail.detail = detail.id
          and level=?
        group by phase, type, detail.id, details
        order by phase
      """, (levid,))
      
      problems = []
      
      for row in cursor.fetchall():
        phase, type, detid, details, ndocs = row
        problems.append(self._getProblem(levid, phase, type, detid, details, ndocs))
      
      if problems:
        yield (level, problems)
      
  def getProblemsByPhase(self):
    """ yield tuples of the form (phase, [{problem}, {problem}...]) """
  
    cursor = self._conn.cursor()

    for phid, phase in enumerate(PHASES):
      cursor.execute("""
        select level, type, detail.id, details, count(document) as ndocs
        from problemtype, detail, documentdetail
        where detail.problemtype = problemtype.id
          and documentdetail.detail = detail.id
          and phase=?
        group by level, type, detail.id, details
        order by level
      """, (phid,))
      
      problems = []
      
      for row in cursor.fetchall():
        level, type, detid, details, ndocs = row
        problems.append(self._getProblem(level, phid, type, detid, details, ndocs))

      if problems:
        yield (phase, problems)
  
  def getProblemSummary(self):
    cursor = self._conn.cursor()
  
    for phid, phase in enumerate(PHASES):
      cursor.execute("""
        select level, type, count(documentdetail.id)
        from problemtype, detail, documentdetail
        where detail.problemtype = problemtype.id
          and documentdetail.detail = detail.id
          and phase = ?
        group by level, type
      """, (phid,))

      # initialize a dictionary with empty counts...
      d = {}
      for level in LEVELS:
        d[level] = ''
        
      for row in cursor.fetchall():
        level, type, count = row
        d[LEVELS[level]] = count

      yield (phase, d)

  def getDocumentProblems(self, docurl):
    cursor = self._conn.cursor()

    cursor.execute("""
      select level, phase, type, details
      from problemtype, detail, documentdetail, document
      where detail.problemtype = problemtype.id
        and documentdetail.detail = detail.id
        and documentdetail.document = document.id
        and document.url = ?
      order by level, phase, type, details
    """, (docurl,))
    
    for row in cursor.fetchall():
      level, phase, type, details = row
      yield {'level': LEVELS[level], 'phase': PHASES[phase], 'type': type, 'details': details}
    
  def _getDocumentSummary(self, docurl):
    """ return a summary (problem counts and actual problems) for a document as a tuple of the form (skip, crit, warn, note, problems) """

    count = {}
    for level in LEVELS:
      count[level] = 0

    problems = list(self.getDocumentProblems(docurl))
    for problem in problems:
      count[problem['level']] += 1
    
    return (count['SKIP'], count['CRIT'], count['WARN'], count['NOTE'], problems)

  def formatProblems(self, problems, level):
    # make a list
    probs = []
    for prob in problems:
      if prob['level'] != level: continue

      if prob['details']:
        s = "%s %s" % (prob['type'], prob['details'])
      else:
        s = prob['type']
      probs.append(s)

    s = '; '.join(probs)
    if len(s) > 80:
      return s[:80] + "..."
    else:
      return s

  def getSuccesfulDocuments(self):
    """ return a list of documents with no problems except for NOTE and SUCC levels """
    
    cursor = self._conn.cursor()
    cursor.execute("""
      select distinct document.id, url
      from document, documentdetail, detail, problemtype
      where documentdetail.document = document.id
        and documentdetail.detail = detail.id
        and detail.problemtype = problemtype.id
        and problemtype.level = ?
      except
      select distinct document.id, url
      from document, documentdetail, detail, problemtype
      where documentdetail.document = document.id
        and documentdetail.detail = detail.id
        and detail.problemtype = problemtype.id
        and problemtype.level = ?
    """, (LEVELS.index('SUCC'), LEVELS.index('WARN')))
    
    l = []
    for row in cursor.fetchall():
      summary = self._getDocumentSummary(row[1])
      notes = self.formatProblems(summary[4], 'NOTE')
      l.append(summary + (self._docs[row[1]], notes))
    l.sort()
    return l

  def getDocumentsWithWarnings(self):
    """ return a list of succesful documents with some warnings """
    
    cursor = self._conn.cursor()
    cursor.execute("""
      select distinct document.id, url
      from document, documentdetail, detail, problemtype
      where documentdetail.document = document.id
        and documentdetail.detail = detail.id
        and detail.problemtype = problemtype.id
        and problemtype.level = ?
      intersect
      select distinct document.id, url
      from document, documentdetail, detail, problemtype
      where documentdetail.document = document.id
        and documentdetail.detail = detail.id
        and detail.problemtype = problemtype.id
        and problemtype.level = ?
    """, (LEVELS.index('SUCC'), LEVELS.index('WARN')))
    
    l = []
    for row in cursor.fetchall():
      summary = self._getDocumentSummary(row[1])
      warnings = self.formatProblems(summary[4], 'WARN')
      l.append(summary + (self._docs[row[1]], warnings))
    l.sort()
    return l

  def getFailedDocuments(self):
    """ return a list of documents with severe problems """
    
    cursor = self._conn.cursor()
    cursor.execute("""
      select document.id, url
      from document, documentdetail, detail, problemtype
      where documentdetail.document = document.id
        and documentdetail.detail = detail.id
        and detail.problemtype = problemtype.id
        and problemtype.level = ?
      except
      select distinct document.id, url
      from document, documentdetail, detail, problemtype
      where documentdetail.document = document.id
        and documentdetail.detail = detail.id
        and detail.problemtype = problemtype.id
        and problemtype.level = ?
    """, (LEVELS.index('CRIT'),LEVELS.index('SKIP')))
    
    l = []
    for row in cursor.fetchall():
      summary = self._getDocumentSummary(row[1])
      errors = self.formatProblems(summary[4], 'CRIT')
      l.append(summary + (self._docs[row[1]], errors))
    l.sort()
    return l
  
  def getSkippedDocuments(self):
    """ yield tuples of the form (document, [{problem}, {problem}...]) for documents that were skipped """
    
    cursor = self._conn.cursor()
    cursor.execute("""
      select document.id, url
      from document, documentdetail, detail, problemtype
      where documentdetail.document = document.id
        and documentdetail.detail = detail.id
        and detail.problemtype = problemtype.id
        and problemtype.level = ?
    """, (LEVELS.index('SKIP'),))
    
    l = []
    for row in cursor.fetchall():
      summary = self._getDocumentSummary(row[1])
      errors = self.formatProblems(summary[4], 'SKIP')
      l.append(summary + (self._docs[row[1]], errors))
    l.sort()
    return l
  
  def getMetadataFields(self, doc):
    try:
      return rdf.METADATA_FIELDS[list(doc.getMetadata(rdf.RDF.type))[0]]
    except IndexError:
      return ()
  
  def getBaseUrl(self):
    url = self._session.getConfig().getBaseURL()
    if url is None: return None
    return url + self._session.getSource().getName() + "/"
    
  def getCGIURL(self):
    return self._session.getConfig().getCGIURL()
  
  def getConfigName(self):
    return self._session.getConfig().getConfigName()

  def getSourceName(self):
    return self._session.getSource().getName()

  def getSourceType(self):
    return self._session.getSource().getType()
